from __future__ import with_statement

import os, sys, cStringIO, codecs

import pyrem_torq.treeseq as ptt
import pyrem_torq.expression as pte
import pyrem_strs.originedunicode as rm

from util_timeout import apply_on_timeout
try:
    import util_threadpool
except: util_threadpool = None
    
from util_simplelogging import SimpleLogging

from pyrem_torq.extra import NodeWithOrigin
from nodeformatter import *
from _prepscript_util import *

def extract_label_usage(expr):
    neglectedLabelSet = frozenset([ "any" ])
    newLabels = []
    referredLabels = []
    for e in pte.inner_expr_iter(expr):
        if hasattr(e, 'extract_new_labels'): newLabels.extend(e.extract_new_labels())
        if hasattr(e, 'extract_labels'): referredLabels.extend(e.extract_labels())
    newLabels = sorted(set(newLabels) - neglectedLabelSet)
    referredLabels = sorted(set(referredLabels) - neglectedLabelSet)
    return newLabels, referredLabels

def label_dependency_check(exprs, justWarning=True, whiteList=None):
    curDefiendLabelSet = set()
    for i, expr in enumerate(exprs):
        label = i + 1
        if isinstance(expr, tuple):
            label = expr[0]; expr = expr[1]
        defined, used = extract_label_usage(expr)
        usedButUndefined = set(used) - curDefiendLabelSet
        usedButUndefined = filter(lambda L: L not in whiteList, usedButUndefined)
        if usedButUndefined:
            message = "undefined label(s) usage: expression=%s, label=%s" % ( str(label), ",".join(sorted(usedButUndefined)) )
            if justWarning: print >> sys.stderr, "warning: %s" % message
            else:
                raise SystemError(message)
            return
        curDefiendLabelSet.update(defined)

DEFAULT_TIMEOUT_VALUE = 300

class __CommandlineParser(object):
    def __init__(self, sys_argv):
        import getopt
        
        self.prepName = None
        self.inputFileNames = []
        self.outputFileNames = []
        self.optionDebugPreprocessTrace = False
        self.optionScriptCheck = False
        self.optionVerbose = False
        self.preprocessorOptions = []
        self.preprocessorOptionsExpanded = []
        self.preprocessorOptionsRaw = []
        self.optionInputDirs = []
        self.optionTimeOut = DEFAULT_TIMEOUT_VALUE
        self.optionNolocation = False
        self.optionThreads = "io" # either "-", "io", or int
        self.optionForce = False
        self.optionDebugPrintStackTrace = False
        self.optionErrorFileLog = None
        self.optionDebugNoFileWrite = False
        
        self.usage = """
usage: prep.py PREP sourcefile...
options
  -d dir: finds source files from the directory.
  -h: shows this message.
  -i filelist: list of input source filess.
  -o output: output file.
  -P: the following command-line arguments are passed to preprocess script.
  -l errorlog: skips a source file, when it contains invalid character or 
      can't be opened, or can't be preprocessed. creates a file errorlog,
      which will contains these file names.
  -v: verbose.
  --checkscript: just reads PREP and checks it.
  --debug-*: debug options. --debug-help to see descriptions.
  --nolocation: don't output line number and column of each token.
  --prepargs=file: reads arguments for preprocess script from the file.
  --rebuild: do preprocess even when preprocessed files exist.
  --timeout=sec: when preprocess of a file don't stop within the time period,
    prints warning messages to log (default %(DEFAULT_TIMEOUT_VALUE)d).
  --threads=tnum: uses up-to tnum worker threads. when tnum is '*1', 
      uses the count of processors. 
  --threads=io: uses extra thread for reading and writing files. (default)
  --threads=-: don't uses extra threads.
"""[1:-1] % (globals())

        self.debugOptionUsage = """
  --debug-preprocess-trace: prints trace of parsing process.
  --debug-nofilewrite: doesn't generate result files (just for profiling).
  --debug-printstacktrace: prints stack trace when some child thread 
      raises an exception (valid when option --threads=tnum is turned on).
"""[1:-1]
    
        if len(sys_argv) == 1:
            print self.usage
            sys.exit(0)
        
        argv = sys_argv
        for i, a in enumerate(sys_argv):
            if a == "-P":
                argv = sys_argv[:i]
                self.preprocessorOptions = sys_argv[i + 1:]
                break # for i
        
        debugOptions = [ "debug-help", 
                "debug-preprocess-trace", "debug-printstacktrace", "debug-nofilewrite" ]
        opts, args = getopt.gnu_getopt(argv[1:], "d:hi:o:vl:", 
                [ "checkscript", "prepargs=", "timeout=", "nolocation", "threads=",
                "rebuild" ] + debugOptions)
        for o, a in opts:
            if o == "-h":
                print self.usage
                sys.exit(0)
            elif o == "-o":
                self.outputFileNames.append(a)
            elif o.startswith("--debug"):
                if o == "--debug-help":
                    print self.debugOptionUsage
                    sys.exit(0)
                if o == "--debug-preprocess-trace":
                    self.optionDebugPreprocessTrace = True
                elif o == "--debug-printstacktrace":
                    self.optionDebugPrintStackTrace = True
                elif o == "--debug-nofilewrite":
                    self.optionDebugNoFileWrite = True
                else:
                    assert False
            elif o == "--checkscript":
                self.optionScriptCheck = True
            elif o == "--prepargs":
                for line in open(a):
                    self.preprocessorOptionsExpanded.append(line.rstrip())
                self.preprocessorOptionsRaw.append("--prepargs=%s" % a)
            elif o == "-v":
                self.optionVerbose = True
            elif o == "-d":
                self.optionInputDirs.append(a)
            elif o == "-i":
                for line in open(a):
                    self.inputFileNames.append(line.rstrip())
            elif o == "--timeout":
                try: value = int(a)
                except ValueError: value = float(a)
                assert value > 0
                self.optionTimeOut = value
            elif o == "--nolocation":
                self.optionNolocation = True
            elif o == "--threads":
                if util_threadpool is None and a != "-":
                    sys.stderr.write("warning> neglect option --threads (use CPython 2.6 or later to use this feature)")
                if a in ( "-", "io" ): 
                    self.optionThreads = a
                else:
                    if a.startswith('*'):
                        num = util_threadpool.theAvailableNativeThreads * int(a[1:])
                    else:
                        num = int(a)
                    self.optionThreads = num
                    if self.optionThreads <= 0:
                        raise SystemError("option --threads arguments must be >= 1.")
                    if not util_threadpool.theVmIsGILFree:
                        sys.stderr.write("warning> option --threads=num is not recommended because of GIL\n")
                    if self.optionThreads > util_threadpool.theAvailableNativeThreads:
                        sys.stderr.write("warning> argument of --threads exceeds available native threads\n")
            elif o == "--rebuild":
                self.optionForce = True
            elif o == "-l":
                self.optionErrorFileLog = a
            else:
                assert False
        
        for a in args:
            if not self.prepName: self.prepName = a
            else:
                self.inputFileNames.append(a)
        
        if not self.prepName: raise SystemError("no preproessor name is given")
        
        if self.optionInputDirs:
            if self.inputFileNames or self.outputFileNames:
                raise SystemError("input/output files must not be specified with option -d")
        else:
            if not self.inputFileNames: raise SystemError("no input file given")
        if self.outputFileNames:
            if len(self.inputFileNames) != len(self.outputFileNames):
                raise SystemError("number of input/output files mismatch")
    
def load_preprocess_script(prepName):
    moduleFunctions = [ "build_whitelist", "build_exprs", "build_nodeformattable", "build_decoder", "build_tokenizer",
        "normalized_option_strs", "get_option_description", "get_version", "get_target_file_predicate" ]
    try:
        prepModule = __import__("prep_" + prepName)
        for f in moduleFunctions:
            if not hasattr(prepModule, f): raise SystemError("module missing function: %s" % f)
    except ImportError: 
        raise SystemError("can't import the preprocessor: %s" % prepName)
    return prepModule

def file_startswith(f, header):
    headerLines = header.split("\n")
    if headerLines and headerLines[-1] == '':
        del headerLines[-1]
    headerLines = [hl.rstrip() for hl in headerLines]
    for hl in headerLines:
        fl = f.readline().rstrip()
        if fl != hl: return False
    return True

def __reraise_pool(pool, exitImmediatelyWhenNotAvailable=False):
    for _, result in (pool.get_iter_nowait if exitImmediatelyWhenNotAvailable else pool.get_iter)():
        if isinstance(result, Exception):
            raise result

class FreeMarkerError(AssertionError): pass

def assert_no_free_marker(expr):
    for e in pte.free_marker_iter(expr):
        raise FreeMarkerError("Free Marker: %s" % e.name)

def __main_i(cmd, errorFileLogQ):
    log_write = sys.stderr.write if cmd.optionVerbose else None
    warning_write = sys.stderr.write
    if util_threadpool is None:
        warning_write("warning> module threading is not available, so force apply --threads=-\n")
    
    def timeout_warning(fileName):
        if cmd.optionTimeOut: warning_write("warning> taking long time for file: %s\n" % fileName )
    
    prepModule = load_preprocess_script(cmd.prepName)
    
    allOptions = cmd.preprocessorOptions + cmd.preprocessorOptionsExpanded
    moduleVersion = prepModule.get_version()
    isTargetFile = prepModule.get_target_file_predicate(allOptions)
    normalizedPrepOptionStrs = prepModule.normalized_option_strs(cmd.preprocessorOptions) + cmd.preprocessorOptionsRaw
    exprs = prepModule.build_exprs(allOptions)
    for exprOrTuple in exprs:
        expr = exprOrTuple if not isinstance(exprOrTuple, tuple) else exprOrTuple[1]
        if isinstance(expr, pte.TorqExpression):
            assert_no_free_marker(expr)
            
    nodeFormatterTable = prepModule.build_nodeformattable(allOptions)
    decoder = prepModule.build_decoder(allOptions)
    tokenizer = prepModule.build_tokenizer(allOptions)
    extraWhiteList = prepModule.build_whitelist(allOptions)
    
    def is_torq_expr(e):
        if isinstance(e, tuple):
            assert len(e) == 2
            return isinstance(e[1], pte.TorqExpression)
        else:
            return isinstance(e, pte.TorqExpression)
        return False
    
    whiteList = tuple([ 'eof' ] + extraWhiteList)
    if cmd.optionScriptCheck:
        label_dependency_check(filter(is_torq_expr, exprs), False, whiteList=whiteList)
        sys.exit(0)
    else:
        label_dependency_check(filter(is_torq_expr, exprs), whiteList=whiteList)
    
    if cmd.optionInputDirs:
        def input_file_names_iter():
            for d in cmd.optionInputDirs:
                for root, dirs, files in os.walk(d):
                    for f in filter(isTargetFile, files):
                        yield os.path.join(root, f)
    else:
        for f in cmd.inputFileNames:
            if not isTargetFile(f):
                raise SystemError("preprocess script rejects an input file: %s" % f)
        def input_file_names_iter():
            for fn in cmd.inputFileNames: yield fn
    
    class ParseError(ValueError): pass
    
    def parse(inputText, inputFileName, exprs):
        labeledExprs = [(expr if isinstance(expr, tuple) else ( None, expr )) \
                for expr in exprs]
        seq = tokenizer(inputText, inputFileName)
        for label, expr in labeledExprs:
            if cmd.optionDebugPreprocessTrace: 
                print "debug> seq=", "\n".join(ptt.seq_pretty(seq))
                if label: print "debug> step: %s" % str(label)
            if isinstance(expr, pte.TorqExpression):
                newSeq = expr.parse(seq)
                if newSeq is None: raise ParseError(label)
                seq = newSeq
            else:
                seq = expr(seq)
        if cmd.optionDebugPreprocessTrace: print "debug> seq=", "\n".join(ptt.seq_pretty(seq))
        return seq
    
    def format_write(header, seq, output):
        output_write = output.write
        output_write(header)
        
        if cmd.optionNolocation: 
            for text, node in format_seq(seq, nodeFormatterTable):
                assert text.find('\n') < 0
                output_write("\t\t%s\n" % text)
        else:
            for text, node in format_seq(seq, nodeFormatterTable):
                assert text.find('\n') < 0
                if isinstance(node, NodeWithOrigin):
                    node = node.original
                if len(node) > 1 and isinstance(node[1], rm.OriginedStr) and node[1]:
                    firstCharPos = node[1].origin0.to_rowcol()[0]
                    if not(isinstance(node[-1], rm.OriginedStr) and node[-1]):
                        warning_write("warning> string/node are mixed at child level of node: %s\n" % node[0])
                        output_write("%s\t\t%s\n" % ("%d,%d" % firstCharPos, text ))
                    else:
                        lastCharPos = node[-1].get_origin()[-1].to_rowcol()[1]
                        output_write("%s\t%s\t%s\n" % ("%d,%d" % firstCharPos, "%d,%d" % lastCharPos, text ))
                else:
                    output_write("\t\t%s\n" % text)
    
    def make_header(md5):
        r = []
        r.append("# generated by prep+%s %s" % ( cmd.prepName, (".".join(("%d" % v) for v in moduleVersion)) ))
        r.append("# option %s" % " ".join(normalizedPrepOptionStrs))
        if cmd.optionNolocation: r.append("# option --nolocation")
        r.append("# md5 %s" % md5)
        assert "".join(r).find('\n') < 0
        return "\n".join(r) + "\n"
    
    def __input_file(inputFileName):
        try:
            f = open(inputFileName, "rb")
            try:
                inputFileBytes = f.read()
            finally: f.close()
        except:
            if not errorFileLogQ: raise
            errorFileLogQ.put("read error: %s" % inputFileName)
            return None
        return inputFileBytes
    
    def __output_file(outputFileName, content, mode="wb", encoding=None):
        assert "w" in mode
        tempfn = outputFileName + ".#tmp#"
        try:
            if encoding:
                output = codecs.open(tempfn, mode, encoding, buffering=0)
            else:
                output = open(tempfn, mode, buffering=0)
            output.write(content)
            output.close()
        except:
            if not errorFileLogQ: raise
            errorFileLogQ.put("write error: %s" % tempfn)
            return
        
        try:
            if os.path.exists(outputFileName): os.remove(outputFileName)
            os.rename(tempfn, outputFileName)
        except:
            if not errorFileLogQ: raise
            errorFileLogQ.put("write error: %s" % outputFileName)
        
    def process_a_file(inputFileName, inputFileBytes, outputFileName, outputFileFunc):
        header = make_header(calc_md5_bytes(inputFileBytes))
        try:
            u = decoder(inputFileBytes, inputFileName)
        except UnicodeDecodeError:
            if not errorFileLogQ: raise ValueError("can't decode text of file: %s\n" % inputFileName)
            errorFileLogQ.put("decode error: %s" % inputFileName)
            return
        inputText = rm.origined_str(rm.NamedStr(inputFileName, u))
        
        if cmd.optionNolocation: inputText = unicode(inputText)
        
        needPreprocessing = True
        if not(cmd.optionForce or cmd.optionDebugPreprocessTrace):
            try:
                if os.path.exists(outputFileName) and file_startswith(open(outputFileName, "rb"), header):
                    needPreprocessing = False
                    log_write and log_write("log> preprocessed file already exists: %s\n" % inputFileName)
            except IOError:
                pass
        
        if needPreprocessing:
            log_write and log_write("log> processing: %s\n" % inputFileName)
            try:
                with apply_on_timeout(cmd.optionTimeOut, timeout_warning, ( inputFileName, )):
                    seq = parse(inputText, inputFileName, exprs)
            except pte.InterpretError:
                if not errorFileLogQ: raise
                errorFileLogQ.put("preprocess error: %s" % inputFileName)
                return
            except ParseError:
                if not errorFileLogQ: raise
                errorFileLogQ.put("preprocess error: %s" % inputFileName)
                return
            
            buf = cStringIO.StringIO()
            format_write(header, seq, buf)
            cmd.optionDebugNoFileWrite or outputFileFunc(outputFileName, buf.getvalue(), "wb")
            buf.close()
    
    def read_and_process_a_file(inputFileName, outputFileName, outputFileFunc):
        inputFileBytes = __input_file(inputFileName)
        if inputFileBytes is None: return # read error
        return process_a_file(inputFileName, inputFileBytes, outputFileName, outputFileFunc)
        
    PST = cmd.optionDebugPrintStackTrace
    def calc_output_file_name(inputFileIndex, inputFileName):
        return cmd.outputFileNames[inputFileIndex] if cmd.outputFileNames else inputFileName + ".prep"
    
    if cmd.optionThreads == "-" or util_threadpool is None:
        for i, inputFileName in enumerate(input_file_names_iter()):
            read_and_process_a_file(inputFileName, calc_output_file_name(i, inputFileName), __output_file)
    elif isinstance(cmd.optionThreads, int):
        assert cmd.optionThreads >= 1
        threadPool = util_threadpool.ThreadPool(cmd.optionThreads,
                queueSize=util_threadpool.INFINITE, printStackTrace=PST)
        for iName, result in threadPool.apply_iter(( read_and_process_a_file, ( iName, calc_output_file_name(i, iName), __output_file ), iName ) \
                for i, iName in enumerate(input_file_names_iter())):
            if isinstance(result, Exception): raise result
        threadPool.join()
    elif cmd.optionThreads == "io":
        prefetchSize = 5
        outpPool = util_threadpool.ThreadPool(1, prefetchSize, printStackTrace=PST)
        inpPool = util_threadpool.ThreadPool(1, prefetchSize, printStackTrace=PST)
        def output_file(name, content, mode="wb", encoding=None):
            outpPool.apply(__output_file, ( name, content, mode, encoding ))
        for i, ( iName, iContent ) in enumerate( \
                inpPool.apply_iter(( __input_file, ( fn, ), fn ) for fn in input_file_names_iter())):
            if iContent is None: continue # read error
            if isinstance(iContent, Exception): raise iContent
            process_a_file(iName, iContent, calc_output_file_name(i, iName), output_file)
            if i % 100 == 0: __reraise_pool(outpPool, exitImmediatelyWhenNotAvailable=True)
        outpPool.join()
        __reraise_pool(outpPool)
    else:
        assert False

def main():
    cmd = __CommandlineParser(sys.argv)
    
    if cmd.optionErrorFileLog:
        lg = SimpleLogging(cmd.optionErrorFileLog)
        with lg as errorFileLogQ:
            __main_i(cmd, errorFileLogQ)
    else:
        lg = SimpleLogging(None)
        with lg as errorFileLogQ:
            __main_i(cmd, None) # exceptions will not be logged, just be raised.
        
    if cmd.optionVerbose:
        sys.stderr.write("info> elapsed time: %f\n" % lg.elapsedtime)

if __name__ == '__main__':
    main()
        
